import json
import re
from datetime import datetime
from collections import defaultdict

print("=== 修复版数据清洗工具 ===")
print("功能: 过滤HTML、二进制数据，保留API数据")
print()

input_file = "数据清洗/extracted_data_20251020_221853.json"

try:
    print("正在读取数据文件...")
    with open(input_file, 'r', encoding='utf-8') as f:
        data = json.load(f)
    print("OK: 数据文件读取成功")
except Exception as e:
    print(f"ERROR: 无法读取数据文件 - {e}")
    input("按回车键退出...")
    exit(1)

# 查找所有requests和responses表
all_requests = []
all_responses = []

tables = data.get('tables', {})
print(f"发现 {len(tables)} 个表:")

for table_name, table_data in tables.items():
    if 'data' in table_data:
        record_count = len(table_data['data'])
        print(f"  {table_name}: {record_count:,} 条记录")

        if 'request' in table_name.lower():
            all_requests.extend(table_data['data'])
            print(f"    -> 添加到requests集合")
        elif 'response' in table_name.lower():
            all_responses.extend(table_data['data'])
            print(f"    -> 添加到responses集合")

print(f"\n=== 数据汇总 ===")
print(f"总requests: {len(all_requests):,} 条")
print(f"总responses: {len(all_responses):,} 条")

if not all_requests:
    print("ERROR: 没有找到请求数据")
    input("按回车键退出...")
    exit(1)

# 确定处理数量
max_process = min(5000, len(all_requests), len(all_responses))
print(f"\n将处理前 {max_process:,} 条记录")

# 统计信息
stats = {
    'processed': 0,
    'html_filtered': 0,
    'binary_filtered': 0,
    'static_filtered': 0,
    'kept': 0,
    'empty_response': 0
}

interfaces = {}

print("开始处理...")

for i in range(max_process):
    try:
        stats['processed'] += 1

        # 进度显示
        if stats['processed'] % 500 == 0:
            print(f"  进度: {stats['processed']:,}/{max_process:,} ({stats['processed']/max_process*100:.1f}%)")

        req = all_requests[i]
        resp = all_responses[i] if i < len(all_responses) else {}

        # 基本字段提取和验证
        method = str(req.get('method', 'GET')).upper().strip()
        if method not in ['GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD', 'OPTIONS']:
            method = 'GET'

        url = req.get('url', req.get('path', '/'))
        if not url:
            url = '/'

        # URL标准化
        url_str = str(url).strip()

        # 如果是完整URL，保持原样
        if url_str.startswith(('http://', 'https://')):
            # 移除查询参数，但保持协议和域名
            if '?' in url_str:
                url_str = url_str.split('?')[0]
            # 移除末尾斜杠（除非是根路径）
            if url_str != '/' and url_str.endswith('/') and len(url_str) > 1:
                url_str = url_str[:-1]
        else:
            # 处理相对路径
            if '?' in url_str:
                url_str = url_str.split('?')[0]
            # 确保以/开头（只对相对路径）
            if not url_str.startswith('/'):
                url_str = '/' + url_str
            # 移除末尾斜杠（除非是根路径）
            if url_str != '/' and url_str.endswith('/'):
                url_str = url_str[:-1]

        interface_key = f"{method}:{url_str}"

        # 获取响应内容
        resp_body = resp.get('body', resp.get('response_body', ''))
        resp_status = resp.get('status_code', resp.get('status', 0))

        # 获取Content-Type
        content_type = ''
        if resp.get('headers'):
            headers = resp['headers']
            if isinstance(headers, dict):
                content_type = headers.get('Content-Type', '').lower()
            elif isinstance(headers, str):
                try:
                    header_dict = json.loads(headers)
                    content_type = header_dict.get('Content-Type', '').lower()
                except:
                    pass

        # 响应内容检查
        should_filter = False
        filter_reason = ''

        if not resp_body:
            stats['empty_response'] += 1
        else:
            resp_str = str(resp_body)

            # HTML检查
            html_indicators = ['<html', '<!DOCTYPE', '<head>', '<body>', '<div>', '<script']
            if any(indicator.lower() in resp_str.lower() for indicator in html_indicators):
                should_filter = True
                filter_reason = 'HTML'
            elif 'text/html' in content_type:
                should_filter = True
                filter_reason = 'HTML (Content-Type)'

            # 二进制检查
            if not should_filter and len(resp_str) > 50:
                # 检查不可打印字符比例
                sample = resp_str[:200]
                unprintable = sum(1 for c in sample if ord(c) < 32 and c not in ['\n', '\r', '\t'])
                if unprintable / len(sample) > 0.2:  # 超过20%不可打印字符
                    should_filter = True
                    filter_reason = '二进制'

            # 静态资源检查
            if not should_filter and content_type:
                static_types = ['text/css', 'application/javascript', 'image/', 'video/', 'audio/', 'font/']
                if any(static_type in content_type for static_type in static_types):
                    should_filter = True
                    filter_reason = '静态资源'

            # 更新统计
            if should_filter:
                if 'HTML' in filter_reason:
                    stats['html_filtered'] += 1
                elif '二进制' in filter_reason:
                    stats['binary_filtered'] += 1
                elif '静态' in filter_reason:
                    stats['static_filtered'] += 1

        if should_filter:
            continue

        stats['kept'] += 1

        # 处理请求headers
        req_headers = {}
        if req.get('headers'):
            if isinstance(req['headers'], dict):
                req_headers = req['headers']
            elif isinstance(req['headers'], str):
                try:
                    req_headers = json.loads(req['headers'])
                except:
                    pass

        # 处理响应headers
        resp_headers = {}
        if resp.get('headers'):
            if isinstance(resp['headers'], dict):
                resp_headers = resp['headers']
            elif isinstance(resp['headers'], str):
                try:
                    resp_headers = json.loads(resp['headers'])
                except:
                    pass

        # 创建接口信息
        interface_info = {
            "interface_key": interface_key,
            "method": method,
            "url": url_str,
            "original_url": str(url),
            "request": {
                "method": method,
                "url": url_str,
                "headers": req_headers,
                "body": req.get('body', req.get('request_body')),
                "params": [],
                "query_params": {}
            },
            "response": {
                "status_code": resp_status,
                "headers": resp_headers,
                "body": resp_body,
                "size": resp.get('size', len(str(resp_body)) if resp_body else 0)
            },
            "metadata": {
                "timestamp": req.get('timestamp', resp.get('timestamp')),
                "extracted_at": datetime.now().isoformat()
            }
        }

        # 去重处理
        if interface_key not in interfaces:
            interfaces[interface_key] = interface_info
        else:
            # 合并信息
            existing = interfaces[interface_key]
            existing['request']['headers'].update(req_headers)
            existing['response']['headers'].update(resp_headers)

            # 更新状态码（优先使用非零状态码）
            if resp_status and resp_status != 0:
                existing['response']['status_code'] = resp_status

    except Exception as e:
        if i < 5:  # 只显示前5个错误
            print(f"  处理第{i}条记录时出错: {e}")
        continue

print(f"\n=== 处理结果 ===")
print(f"处理记录: {stats['processed']:,}")
print(f"  - HTML过滤: {stats['html_filtered']:,}")
print(f"  - 二进制过滤: {stats['binary_filtered']:,}")
print(f"  - 静态资源过滤: {stats['static_filtered']:,}")
print(f"  - 空响应: {stats['empty_response']:,}")
print(f"保留记录: {stats['kept']:,}")
print(f"唯一接口: {len(interfaces):,}")

# 计算过滤率
if stats['processed'] > 0:
    total_filtered = stats['html_filtered'] + stats['binary_filtered'] + stats['static_filtered']
    filter_rate = (total_filtered / stats['processed']) * 100
    print(f"过滤率: {filter_rate:.1f}%")

# 生成统计
method_stats = defaultdict(int)
status_stats = defaultdict(int)

for interface in interfaces.values():
    method_stats[interface['method']] += 1
    status = str(interface['response']['status_code'])
    status_stats[status] += 1

# 保存结果
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
output_file = f"数据清洗/fixed_cleaned_{timestamp}.json"

output_data = {
    "metadata": {
        "cleaned_at": datetime.now().isoformat(),
        "source_file": input_file,
        "total_interfaces": len(interfaces),
        "processed_records": stats['processed'],
        "cleaning_version": "fixed_1.0"
    },
    "filtering_stats": stats,
    "interfaces": list(interfaces.values()),
    "summary": {
        "methods": dict(method_stats),
        "status_codes": dict(status_stats)
    }
}

try:
    with open(output_file, 'w', encoding='utf-8') as f:
        json.dump(output_data, f, indent=2, ensure_ascii=False)
    print(f"✅ 结果已保存: {output_file}")
except Exception as e:
    print(f"❌ 保存失败: {e}")

print(f"\n=== 接口示例 (前5个) ===")
for i, (key, interface) in enumerate(list(interfaces.items())[:5]):
    print(f"{i+1}. {interface['method']} {interface['url']} -> {interface['response']['status_code']}")

print(f"\n🎉 修复版清洗完成!")
print(f"📊 方法分布: {dict(method_stats)}")
print(f"📊 状态码分布: {dict(list(status_stats.items())[:5])}...")  # 只显示前5个

input("按回车键退出...")